def get_train_data_lyric():
    data = []
    with open('./output.txt', 'r', encoding='utf-8') as f:
        poetry_corpus = f.read()
        print('len poetry', len(poetry_corpus))
        for i in range(len(poetry_corpus)):
            if i % 48 == 0:
                first = poetry_corpus[i:i + 6]
                second = poetry_corpus[i + 6:i + 12]
                assert len(first) == len(second)
                data.append((first, second))
        # input(1)
    return data
from io_util import get_data
#train_ds = get_data(r'C:\Users\xu\Downloads\9d3f0f41-c7df-4211-a32a-9e1fc74f8a68\train')
train_ds = get_data(r'law_train')

def get_law_data():
    datas = []
    for d in train_ds:
        q = d['question']+" 给出选项："
        ans = '选项：'.join(d['candidate_answer'])
        a = d['answer']
        datas.append((q+ans, a))
    return datas
def texts2vocab_list(texts,tokenizer):
    ws_ = []
    for t in texts:
        ws = tokenizer(t)
        ws_.extend(ws)
    return list(set(ws_))